#!/bin/sh
#
# Run test builds on one or more PCP machines
#

_usage()
{
    echo >&2 "Usage: `basename $0` [options] vm ..."
    echo >&2 "options:"
    echo >&2 "  -a          all except shutdown VM [implies -pmiqh]"
    echo >&2 "  -b branch   pcp branch to checkout [default $pcp_branch]"
    echo >&2 "  -c          check git repositories"
    echo >&2 "  -d          output debugging diagnostics"
    echo >&2 "  -e file     prerun script"
    echo >&2 "  -f          force shutdown"
    echo >&2 "  -g what     QA tests or groups to run, e.g. -g \"123 -g sanity -x remote\""
    if [ "$what" = all ]
    then
	echo >&2 "              [default <nothing> => all tests]"
    else
	echo >&2 "              [default $what]"
    fi
    echo >&2 "  -h          harvest QA failures"
    echo >&2 "  -i          install built pcp packages"
    echo >&2 "  -l          harvest daily.log/daily.qa.log after aborted run"
    echo >&2 "  -m          build (Makepkgs) for pcp"
    echo >&2 "  -p          git pull for pcp"
    echo >&2 "  -q          run qa"
    echo >&2 "  -r          rerun qa for failing cases"
    echo >&2 "  -s          shutdown VM"
    echo >&2 "  -S          single-thread, not parallel"
    echo >&2 "  -t          tree for pcp [default $pcp_tree]"
    echo >&2 "default: -pmiqhs"
    echo >&2 "vmlist1: $list_1"
    echo >&2 "vmlist2: $list_2"
    echo >&2 "[both vmlists run in parallel for the default case without -S]"
}

# $1 is "daily" (for build/install) else "daily.qa" (for qa)
#
_script_header()
{
    cat <<End-of-File
#!/bin/sh
#
# PCP Build Script from pcp-daily on `date`
#

[ -f \$HOME/.dailyrc ] && . \$HOME/.dailyrc
if [ $debug = true ]
then
    echo PATH=\$PATH
    echo MAKE=\$MAKE
    which \${MAKE:-make}
fi

tmp=/var/tmp/\$\$
trap "rm -f \$tmp.*; exit 0" 0 1 2 3 15
End-of-File
    if $lflag
    then
	:
    else
	cat <<End-of-File
rm -f \$HOME/$1.log
echo "--- starting $1 on \`date\` ---" >\$HOME/$1.log
if [ -x ./freespace ] 
then
    ./freespace -l >>\$HOME/$1.log
fi
End-of-File
    fi
    if [ "$1" = daily ]
    then
	# not QA, need git tree
	#
	if [ "$pcp_tree" = pcpfans ]
	then
	    cat <<End-of-File
if [ ! -d $build_dir ]
then
    git clone git://sourceware.org/git/pcpfans.git $build_dir
    echo "Info: clone pcpfans git tree" | tee -a \$HOME/daily.log
fi
End-of-File
	else
	    cat <<End-of-File
if [ ! -d $build_dir ]
then
    git clone ssh://bozo/home/kenj/git-mirror/pcp.git $build_dir
    echo "Info: clone $pcp_tree git tree" | tee -a \$HOME/daily.log
fi
End-of-File
	fi
	cat <<End-of-File
cd $build_dir
End-of-File
    fi
}

_script_git()
{
    cat <<End-of-File
git branch -a >\$tmp.branch
if grep " $pcp_branch\$" <\$tmp.branch >/dev/null
then
    :
else
    if grep " remotes/origin/$pcp_branch\$" <\$tmp.branch >/dev/null
    then
	git branch --track $pcp_branch remotes/origin/$pcp_branch
	echo "Info: pcp git branch $pcp_branch set up for remote tracking" | tee -a \$HOME/daily.log
    else
	echo "Error: pcp git branch $pcp_branch unknown!" | tee -a \$HOME/daily.log
	exit 1
    fi
fi
End-of-File
}

_script_git_check()
{
    cat <<End-of-File
git status -s | grep 'M ' >\$tmp.status
if [ -s \$tmp.status ]
then
    echo "Warning: modified pcp files ..." | tee -a \$HOME/daily.log
    sed -e 's/.*M  */    /' \$tmp.status | tee -a \$HOME/daily.log
else
    echo "Info: no modified pcp files" | tee -a \$HOME/daily.log
fi
End-of-File
}

_script_git_pull()
{
    cat <<End-of-File
# need to clean before pulling in case unwanted files no longer removed,
# e.g. in PCP 3.x -> PCP 4.0 migration of source base
# not needed any longer - 26 Mar 2013
#\${MAKE:-make} clean >>\$HOME/daily.log 2>&1
# only VERSION.pcp is expected to be modified, sometimes!
mv VERSION.pcp VERSION.tmp
git checkout -- VERSION.pcp
if git checkout $pcp_branch >>\$HOME/daily.log 2>&1
then
    echo "Info: pcp git checkout $pcp_branch" | tee -a \$HOME/daily.log
    rm VERSION.tmp
else
    echo "Error: pcp git checkout $pcp_branch failed!" | tee -a \$HOME/daily.log
    if [ -f VERSION.pcp ]
    then
	rm VERSION.tmp
    else
	mv VERSION.tmp VERSION.pcp
    fi
    exit 1
fi
# on some VMs, there is a problem with the shutdown that deletes
# some files which causes the subsequent Makepkgs to fail ... try
# to recover from this
git status -s | awk '\$1 == "D" { print \$2 }' | xargs git checkout
if git pull >>\$HOME/daily.log 2>&1
then
    echo "Info: pcp git pull" | tee -a \$HOME/daily.log
else
    echo "Error: pcp git pull failed!" | tee -a \$HOME/daily.log
    exit 1
fi
End-of-File
}

_script_make()
{
    cat <<'End-of-File'
if [ -f qa/admin/list-packages ]
then
    qa/admin/list-packages -m >$tmp.tmp
    if [ -s $tmp.tmp ]
    then
	echo "Info: list-packages reports missing: `cat $tmp.tmp` " | tee -a $HOME/daily.log
    fi
fi
ok=false
# if needed, bump build number for pcp-daily
if [ ! -f VERSION.pcp.daily ]
then
    cp VERSION.pcp VERSION.pcp.daily
else
    diff VERSION.pcp VERSION.pcp.daily >$tmp.tmp
    if egrep 'PACKAGE_MAJOR|PACKAGE_MINOR|PACKAGE_REVISION' <$tmp.tmp >/dev/null
    then
	# different PCP release, do an initial build
	cp VERSION.pcp VERSION.pcp.daily
    else
	# same PCP release, bump away
	awk -F= <VERSION.pcp.daily '
BEGIN	{ OFS = "=" }
$1 == "PACKAGE_BUILD"	{ print "PACKAGE_BUILD",$2+1; next }
			{ print }' \
| sed -e 's/=$//' >$tmp.version
	cp $tmp.version VERSION.pcp.daily
    fi
fi
cp VERSION.pcp.daily VERSION.pcp
# if we're using systemctl, then need to stop PCP services
# to prevent autorestart banging its head against the wall
# once the libraries are removed and the packages are being
# built (and installed)
#
if which systemctl >/dev/null 2>&1
then
    echo "Info: stopping PCP services" | tee -a $HOME/daily.log
    for svc in pmie pmlogger pmproxy pmcd
    do
	sudo systemctl stop $svc
    done
fi
# simulate a first-time build
sudo rm -rf /usr/include/pcp
sudo rm -f /usr/lib*/libpcp[_.]*
# clean up any build turds
rm -f qa/qa_outfiles
End-of-File
    echo "if MAKE=\$MAKE ./Makepkgs --clean >>\$HOME/daily.log 2>&1"
    cat <<'End-of-File'
then
    ok=true
else
    # for Debian/Ubuntu, it may just be the signing of the packages that
    # failed
    #
    if tail -20 $HOME/daily.log | grep 'dpkg-buildpackage: warning: Failed to sign' >/dev/null
    then
	echo "Warning: Makepkgs failed to sign dpkg's" | tee -a $HOME/daily.log
    else
	echo "Error: Makepkgs failed!" | tee -a $HOME/daily.log
	# restore build number
	rm -f VERSION.pcp
	git checkout -- VERSION.pcp
	exit 1
    fi
fi
# restore build number
rm -f VERSION.pcp
git checkout -- VERSION.pcp
# Look for errors/warnings in the build logs ...
#
# In the seds below are for "errors" and "warnings" that are:
# (a) out of out control (e.g. in flex/bison generated code,
#     Qt headers, non-PCP libraries, etc), or
# (b) apparently benign things we're no going to fix, or
# (c) stuff we've triaged and made a "no change needed" call
#
for log in Logs/*
do
    echo "--- $log ---" >>$HOME/daily.log
    cat $log >>$HOME/daily.log
    echo "--- end $log ---" >>$HOME/daily.log
done
grep 'Permission denied' Logs/pcp >$tmp.err
grep ' [Ee]rror: ' Logs/pcp \
| sed >>$tmp.err \
    -e '/^\/usr\/share\/selinux\/devel\/.* duplicate definition of /d' \
    # end
sed -e 's/^/ /' <Logs/pcp \
| grep -i ' warning: ' \
| sed >>$tmp.err \
    -e 's/^ //' \
    -e '/yyunput.* defined but not used/d' \
    -e '/input.* defined but not used/d' \
    -e '/jobserver unavailable: using -j1/d'\
    -e '/^warning: File listed twice: .*python/d' \
    -e '/Deprecated external dependency generator is used!/d' \
    -e '/^dpkg-shlibdeps: warning: .*\/pmda_pmcd.so contains an unresolvable .* probably a plugin/d' \
    -e '/^dpkg-shlibdeps: warning: package could avoid a useless dependency/d' \
    -e '/^configure: WARNING: unrecognized options: --disable-dependency-tracking/d' \
    -e '/QtCore\/qvector.h:.* memcpy(/d' \
    -e '/libpython.* warning: warning: tmpnam() possibly used unsafely/d' \
    -e '/libpython.* warning: warning: tempnam() possibly used unsafely/d' \
    -e '/WARNING: missing directory entry for <.*pcp/s/WARNING/Warning/' \
    -e '/WARNING: missing directory entry for </d' \
    -e '/^Project WARNING: CONFIG-=import_qpa_plugin is deprecated./d' \
    -e '/^Project MESSAGE: Warning: unknown QT: widgets/d' \
    -e '/^Project MESSAGE: Warning: unknown QT: printsupport/d' \
    -e '/ldconfig: Warning: ignoring configuration file that cannot be opened: .*pcp.*BUILDROOT/d' \
    -e '/<stdout>:.* warning: comparison between signed and unsigned/d' \
# end
if [ -s $tmp.err ]
then
    echo "Warning: `wc -l <$tmp.err | sed -e 's/ //g'` build errors/warnings" | tee -a $HOME/daily.log
    cat $tmp.err >>$HOME/daily.log
    echo "--- end build errors/warnings ---" >>$HOME/daily.log
    ok=false
fi
$ok && echo "Info: Makepkgs" | tee -a $HOME/daily.log
End-of-File
}

_script_install()
{
    cat <<'End-of-File'
# extract buildversion
. ./VERSION.pcp
if [ -z "$PACKAGE_MAJOR" -o -z "$PACKAGE_MINOR" -o -z "$PACKAGE_REVISION" ]
then
    ls -l VERSION.pcp | tee -a $HOME/daily.log
    cat VERSION.pcp | tee -a $HOME/daily.log
    echo "Error: VERSION.pcp is damaged" | tee -a $HOME/daily.log
    exit 1
fi
buildversion=$PACKAGE_MAJOR.$PACKAGE_MINOR.$PACKAGE_REVISION
# find lsm ... in pcp-$buildversion for most build types, but in 
# build/deb/pcp-$buildversion for debian
lsm=''
[ -f pcp-$buildversion/pcp.lsm ] && lsm=pcp-$buildversion/pcp.lsm
[ -f build/deb/pcp-$buildversion/pcp.lsm ] && lsm=build/deb/pcp-$buildversion/pcp.lsm
if [ -z "$lsm" ]
then
    ls -l pcp.lsm* pcp-$buildversion/pcp.lsm* | tee -a $HOME/daily.log
    echo "Error: pcp.lsm not found" | tee -a $HOME/daily.log
    exit 1
fi
src_version=`sed -n <$lsm -e 's/[ 	]*$//' -e '/^Version:/{
s/Version:[ 	]*//
p
q
}'`
notfound=true
case `echo build/deb/pcp_[0-9]*.deb`
in
    *\ *)
	ls -l build/deb/pcp_[0-9]*.deb
	echo "Error: more than one deb pkg found" | tee -a $HOME/daily.log
	exit 1
	;;
    'build/deb/pcp_[0-9]*.deb')
	# no deb packages
	;;
    *)
	# Debian packaging ...
	apt_ver=`apt-get -v | sed -n -e '/^apt/{
s/apt //
s/ .*//p
}'`
	case "$apt_ver"
	in
	    #0.*|1.0.*)
	    *)
		# the old-school dpkg way
		#
		if yes Y | sudo dpkg -i --force-depends build/deb/*.deb >>$HOME/daily.log 2>&1
		then
		    echo "Info: dpkg install" | tee -a $HOME/daily.log
		else
		    echo "Error: dpkg failed!" | tee -a $HOME/daily.log
		    exit 1
		fi
		;;
	    #*)
	    do-not-do-this-any-more)
		# this does not work reliably ...
		#
		# the new-school apt-get way
		#
		here=`pwd`
		cd build/deb
		if sudo apt-get --yes --reinstall install ./*.deb >>$HOME/daily.log 2>&1
		then
		    echo "Info: apt-get install" | tee -a $HOME/daily.log
		else
		    echo "Error: apt-get failed!" | tee -a $HOME/daily.log
		    exit 1
		fi
		cd $here
		;;
	esac
	notfound=false
	;;
esac
$notfound && case `echo pcp-$buildversion/build/rpm/pcp-[0-9]*.src.rpm`
in
    *\ *)
	ls -l pcp-$buildversion/build/rpm/pcp-[0-9]*.src.rpm | tee -a $HOME/daily.log
	echo "Error: more than one rpm pkg found" | tee -a $HOME/daily.log
	exit 1
	;;
    "pcp-$buildversion"'/build/rpm/pcp-[0-9]*.src.rpm')
	# no rpm source package, assume some other sort of packaging
	;;
    *)
	# RPM packaging ...
	# Notes may need to add these flags ...
	#  --nodeps	needed because perl-Spreadsheet-Read is not available
	#		e.g. Fedora 15
	#  --oldpackage	sometimes we do a downgrade on the QA machines
	#
	here=`pwd`
	if [ ! -d pcp-$buildversion/build/rpm ]
	then
	    echo "Error: pcp-$buildversion/build/rpm: no dir: rpm failed!" | tee -a $HOME/daily.log
	    exit 1
	fi
	cd pcp-$buildversion/build/rpm
	if sudo rpm -U --force `./cherrypick 2>$tmp.err` >$tmp.out 2>&1
	then
	    cat $tmp.out >>$HOME/daily.log
	    cat $tmp.err >>$HOME/daily.log
	    echo "Info: rpm install" | tee -a $HOME/daily.log
	else
	    cat $tmp.out >>$HOME/daily.log
	    cat $tmp.err >>$HOME/daily.log
	    grep -v 'is already installed' $tmp.out >$tmp.tmp
	    if [ -s $tmp.tmp ]
	    then
		echo "Error: rpm failed!" | tee -a $HOME/daily.log
		exit 1
	    else
		echo "Warning: rpm packages already installed" | tee -a $HOME/daily.log
	    fi
	fi
	cd $here
	notfound=false
	;;
esac
$notfound && if [ -f pcp-$buildversion/build/mac/pcp-[0-9]*[0-9].dmg ]
then
    # Mac OS X "pkg in a disk image" packaging
    here=`pwd`
    cd pcp-$buildversion/build/mac
    if ./cmdline-install >>$HOME/daily.log 2>&1
    then
	echo "Info: installer install" | tee -a $HOME/daily.log
    else
	echo "Error: installer failed!" | tee -a $HOME/daily.log
	exit 1
    fi
    cd $here
elif [ -f pcp-$buildversion/build/sun/pcp-$src_version ]
then
    # System V style pkgadd/pkgrm packaging ...
    here=`pwd`
    cd pcp-$buildversion/build/sun
    yes Y | sudo /usr/sbin/pkgrm pcp >>$HOME/daily.log 2>&1
    if yes Y | sudo /usr/sbin/pkgadd -d `pwd` pcp all >>$HOME/daily.log 2>&1
    then
	echo "Info: pkgadd install" | tee -a $HOME/daily.log
    else
	echo "Error: pkgadd failed!" | tee -a $HOME/daily.log
	exit 1
    fi
    cd $here
    sudo /usr/sbin/svcadm enable -s svc:/application/pcp/pmcd >>$HOME/daily.log 2>&1
    sudo /usr/sbin/svcadm enable -s svc:/application/pcp/pmlogger >>$HOME/daily.log 2>&1
elif [ -f pcp-$buildversion/build/tar/pcp-[0-9]*[0-9].tar.gz ]
then
    # tarball packaging ... (this has to be last, because some of the
    # other packaging regimes also create tarballs)
    here=`pwd`
    tarball=$here/pcp-$buildversion/build/tar/pcp-[0-9]*[0-9].tar.gz
    cd pcp-$buildversion/build/tar
    sudo ./preinstall >>$HOME/daily.log 2>&1
    cd /
    if sudo tar -zxpf $tarball >>$HOME/daily.log 2>&1
    then
	echo "Info: tar install" | tee -a $HOME/daily.log
    else
	echo "Error: tar install failed!" | tee -a $HOME/daily.log
	exit 1
    fi
    cd $here
    cd pcp-$buildversion/build/tar
    sudo ./postinstall >>$HOME/daily.log 2>&1
    cd $here
else
    echo "Don't know how to install packages" | tee -a $HOME/daily.log
    exit 1
fi
if [ ! -f /etc/pcp.env ]
then
    echo "Error: /etc/pcp.env not installed" | tee -a $HOME/daily.log
    exit 1
fi
. /etc/pcp.env
if [ -f pcp-$buildversion/build/sun/pcp-$src_version ]
then
    : pmcd started above
else
    if sudo $PCP_RC_DIR/pcp start >$tmp.out 2>&1
    then
	cat $tmp.out >>$HOME/daily.log
	echo "Info: pmcd started" | tee -a $HOME/daily.log
    else
	cat $tmp.out >>$HOME/daily.log
	echo "Error: pmcd start failed" | tee -a $HOME/daily.log
	exit 1
    fi
fi
running_version=`pmprobe -v pmcd.version | sed -e 's/"//g' -e 's/pmcd.version 1 //'`
echo "Source version: $src_version" >>$HOME/daily.log
echo "Running version: $running_version" >>$HOME/daily.log
if [ "$src_version" != "$running_version" ]
then
    pcp >>$HOME/daily.log
    echo "Error: version mismatch src=$src_version running=$running_version" | tee -a $HOME/daily.log
    exit 1
fi
# sample PMDA may have an updated PMNS, so re-install just in case
( cd $PCP_VAR_DIR/pmdas/sample; sudo ./Install </dev/null ) >>$HOME/daily.log
End-of-File
}

_script_qa()
{
    cat <<End-of-File
rm -f *.out.bad
./changeversion >/dev/null 2>&1

# Policy for QA Farm: exclude all tests tagged BAD
#
qa_restrict="$check_what -x BAD"

echo "Info: QA \$qa_restrict starting"
./check -l \$qa_restrict >>\$HOME/daily.qa.log 2>&1
sts=\$?
# look for badness in daily.qa.log and report here (so it goes into any
# cron-based email)
awk <\$HOME/daily.qa.log '
/waiting for lock/		{ print }
/could not acquire lock/	{ print }
/error while loading shared libraries/	{ print }
\$1 == "Aborted!"		{ print }
\$1 == "Check!"			{ print }'
# get last block from check.log ... then get the summary on
# stdout (not needed in the daily.qa.log already there)
sed -e '/^    [0-9 ]*\$/d' <check.log \
| awk '
NF == 0	{ txt = ""; next }
	{ if (txt == "") txt = \$0
	  else txt = txt "\n" \$0
	}
END	{ print txt }' \
| awk '
\$1 == "Failures:"	{ want = 1 }
\$1 == "Passed"		{ want = 1 }
want == 1		{ print }'
if [ "\$sts" = 0 ]
then
    echo "Info: QA \$qa_restrict done" | tee -a \$HOME/daily.qa.log
else
    echo "Error: QA \$qa_restrict failed!" | tee -a \$HOME/daily.qa.log
fi
# if we're using systemctl, then need to start enabled PCP
# services so we leave things in a good state
#
if which systemctl >/dev/null 2>&1
then
    systemctl list-unit-files >\$tmp.systemctl
    for svc in pmcd pmproxy pmie pmlogger_daily_report pmlogger
    do
	if grep "\$svc.service[ 	]*enabled" <\$tmp.systemctl >/dev/null
	then
	    echo "Info: restarting \$svc" | tee -a \$HOME/daily.qa.log
	    sudo systemctl reset-failed \$svc
	    sudo systemctl restart \$svc
	fi
    done
fi
date >>\$HOME/daily.qa.log
End-of-File
}

_script_qa_rerun()
{
    cat <<End-of-File
if ./recheck >>\$HOME/daily.qa.log 2>&1
then
    echo "Info: QA recheck" | tee -a \$HOME/daily.qa.log
else
    echo "Error: QA recheck failed!" | tee -a \$HOME/daily.qa.log
fi
End-of-File
}

_script_harvest()
{
    cat <<End-of-File
rm -f \$HOME/daily.harvest
touch \$HOME/daily.harvest
nbad=0
nfull=0
for bad in *.out.bad
do
    [ "\$bad" = '*.out.bad' ] && continue
    echo \$bad >>\$HOME/daily.harvest
    nbad=\`expr \$nbad + 1\`
    seq=\`echo \$bad | sed -e 's/\.out\.bad//'\`
    [ -f \$seq.out ] && echo \$seq.out >>\$HOME/daily.harvest
    if [ -f \$seq.full ]
    then
	echo \$seq.full >>\$HOME/daily.harvest
	nfull=\`expr \$nfull + 1\`
    fi
done
[ -s \$HOME/daily.harvest ] && cat \$HOME/daily.harvest >>\$HOME/daily.qa.log
echo "Info: harvest \$nbad *.bad files, \$nfull *.full files" | tee -a \$HOME/daily.qa.log
End-of-File
}

# Mainline starts here ...
#

tmp=/var/tmp/$$
sts=0
trap "rm -f $tmp.*; exit \$sts" 0 1 2 3 15

list_1="vm03 vm01 vm04 vm07 vm12 vm14 vm16 vm18 vm20 vm22 vm24 vm26 vm28 vm30 vm32 vm34"
list_2="bozo vm02 vm00 vm05 vm11 vm15 vm19 vm21 vm23 vm25 vm27 vm29 vm31 vm33 vm37 vm38"

args="$*"
pcp_branch=master
pcp_tree=pcp
build_user=${USER-kenj}
build_dir=src/$pcp_tree
qa_user=${USER-kenj}
qa_dir=src/pcp/qa
cflag=false
fflag=false
hflag=false
iflag=false
lflag=false
mflag=false
pflag=false
qflag=false
rflag=false
what="-g sanity -g libpcp -g pdu -g pmda -g valgrind -g pmie"
what=all
sflag=false
Sflag=false
options=false
work=false
efile=''
debug=false
while getopts "ab:B:cde:fg:hilmpqrsSt:T:?" c
do
    case $c
    in
	a)
	    options=true
	    work=true
	    pflag=true
	    mflag=true
	    iflag=true
	    qflag=true
	    hflag=true
	    ;;
	b)
	    pcp_branch="$OPTARG"
	    ;;
	c)
	    options=true
	    work=true
	    cflag=true
	    ;;
	d)
	    debug=true
	    ;;
	e)
	    efile="$OPTARG"
	    if [ -f "$efile" ]
	    then
		:
	    else
		echo "Error: $efile: cannot be found"
		sts=1
		exit
	    fi
	    ;;
	f)
	    fflag=true
	    ;;
	g)
	    what="$OPTARG"
	    ;;
	h)
	    options=true
	    work=true
	    hflag=true
	    ;;
	i)
	    options=true
	    work=true
	    iflag=true
	    ;;
	l)
	    options=true
	    work=true
	    lflag=true
	    ;;
	m)
	    options=true
	    work=true
	    mflag=true
	    ;;
	p)
	    options=true
	    work=true
	    pflag=true
	    ;;
	q)
	    options=true
	    work=true
	    qflag=true
	    ;;
	r)
	    options=true
	    work=true
	    rflag=true
	    ;;
	s)
	    options=true
	    sflag=true
	    ;;
	S)
	    Sflag=true
	    ;;
	t)
	    pcp_tree="$OPTARG"
	    build_dir=src/$pcp_tree
	    ;;
	?)
	    _usage
	    sts=1
	    exit
	    ;;
    esac
done
shift `expr $OPTIND - 1`

if [ $# -eq 0 ]
then
    if $Sflag
    then
	# serial run
	set -- $list_1 $list_2
    else
	# run in parallel
	set -- $list_1
	pcp-daily $args $list_2 &
    fi
fi

if $options
then
    :
else
    # defaults
    #
    work=true
    pflag=true
    mflag=true
    iflag=true
    qflag=true
    hflag=true
    sflag=true
fi

today=`date +%Y-%m-%d`
mkdir -p $HOME/Logs/by-date/$today

rm -f $tmp.domains

# some driver host specific things ...
#
case `hostname -s`
in
    bozo)
	echo "bozo" >>$tmp.domains
	echo "bozo-vm" >>$tmp.domains
	# echo "fuji" >>$tmp.domains
	# and open up the X server
	#
	# Ubuntu up to 15.04
	#DISPLAY=localhost:0.0 xhost + >/dev/null
	# Ubuntu 15.10
	DISPLAY=:0.0 xhost + >/dev/null
	;;
esac

if which virsh >/dev/null 2>&1
then
    # sudo here is odd ... after Ubuntu 17.10 upgrade, virsh run
    # from a cron-launched shell produces no output, sudo virsh does
    # produce output again ... sigh.
    #
    sudo virsh list --all \
    | sed >>$tmp.domains \
        -e 1d \
        -e '/-----/d' \
        -e 's/^  *//' \
        -e 's/^[0-9][0-9]*[ 	]*//' \
        -e 's/^-[ 	]*//' \
	-e 's/[    ].*//' \
        -e '/^$/d'
fi

for target
do
    grep "^$target" <$tmp.domains >$tmp.domain
    if [ ! -s $tmp.domain ]
    then
	echo "No domain matches \"$target\" ... the domains I have are ..."
	cat $tmp.domains
	continue
    fi
    count=`wc -l <$tmp.domain | sed -e 's/ //g'`
    if [ $count != 1 ]
    then
	echo "Warning: more than one domain matches \"$target\" ... the matches I have are ..."
	cat $tmp.domain
	echo "... using the first one."
    fi
    domain=`sed -e 1q $tmp.domain`
    # used to be vmNN-osname&version, but now just vmNN ...
    # old sed now does nothing
    #
    host=`echo $domain | sed -e '/^vm/s/-.*//'`
    echo "::: $domain :::"

    check_what="$what"
    [ X"$what" = Xall ] && check_what=''
    case "$domain"
    in

	vm01)
	    # 739 segv's sometimes deep inside Python
	    [ X"$what" = Xall ] && check_what='-X 739'
	    ;;

	vm05)
	    # Exclude -x valgrind ... does not work on my Gentoo at the
	    # moment
	    #
	    [ X"$what" = Xall ] && check_what='-x valgrind'
	    ;;

	vm06)
	    # Only selected groups at this stage for FreeBSD 10.4
	    #	sanity trace libpcp kernel pmrep threads
	    #
	    [ X"$what" = Xall ] && check_what='-g sanity -g trace -g libpcp -g kernel -g pmrep -g threads -x valgrind'
	    ;;

	vm09)
	    # Only selected groups at this stage for NetBSD 6.1.5
	    #	sanity trace libpcp kernel pmrep threads
	    #
	    [ X"$what" = Xall ] && check_what='-g sanity -g trace -g libpcp -g kernel -g pmrep -g threads -x valgrind'
	    ;;

	vm10)
	    # Only selected groups at this stage for FreeBSD 10.4
	    #	sanity trace libpcp kernel pmrep threads
	    #
	    [ X"$what" = Xall ] && check_what='-g sanity -g trace -g libpcp -g kernel -g pmrep -g threads -x valgrind'
	    ;;

	vm13)
	    # Only -g sanity at this stage for OpenIndiana
	    #
	    [ X"$what" = Xall ] && check_what='-g sanity'
	    ;;

	vm15)
	    # 1123 loops forever lost in an X11 wasteland
	    [ X"$what" = Xall ] && check_what='-X 1123'
	    ;;

	vm20)
	    # Run from testsuite, not git tree
	    #
	    qa_dir=/var/lib/pcp/testsuite
	    qa_user=pcpqa
	    ;;

	vm32)
	    # Only selected groups at this stage for FreeBSD 11.1
	    #	sanity trace libpcp kernel pmrep threads
	    # and some tests are not working yet
	    #
	    [ X"$what" = Xall ] && check_what='-g sanity -g trace -g libpcp -g kernel -g pmrep -g threads -x valgrind -X 603'
	    ;;

	vm33)
	    # Only selected groups at this stage for OpenBSD 6.3
	    #	sanity trace libpcp kernel pmrep threads
	    #
	    [ X"$what" = Xall ] && check_what='-g sanity -g trace -g libpcp -g kernel -g pmrep -g threads'
	    ;;

	vm35)
	    # this is PCP 2.7.8, need to run as user pcpqa and just
	    # the sanity group and no -p or -m or -i flags
	    # and no -e processing
	    [ X"$what" = Xall ] && check_what='-g sanity'
	    qa_user=pcpqa
	    qa_dir=/var/lib/pcp/testsuite/qa
	    if $pflag
	    then
		echo "Info: skip -p for vm35"
		pflag=false
	    fi
	    if $mflag
	    then
		echo "Info: skip -m for vm35"
		mflag=false
	    fi
	    if $iflag
	    then
		echo "Info: skip -i for vm35"
		iflag=false
	    fi
	    if [ -n "$efile" ]
	    then
		echo "Info: no -e $efile for vm35"
		efile=''
	    fi
	    ;;

	vm37)
	    # Only selected groups this stage for OpenBSD 6.4
	    #	sanity threads kernel
	    # and some tests are not working yet
	    #
	    [ X"$what" = Xall ] && check_what='-g sanity -g threads -g kernel'
	    ;;

	vm38)
	    # Only -g sanity at this stage for NetBSD 8.0
	    #
	    [ X"$what" = Xall ] && check_what='-g sanity'
	    ;;

    esac

    if which virsh >/dev/null 2>&1
    then
	# see note above re. sudo virsh
	#
	state=`sudo virsh list --all | grep "$domain" | awk '{print $3}'`
	case "$state"
	in
	    shut)
		if $work
		then
		    sudo virsh start $domain
		    state=running
		fi
		;;
	    running)
		;;
	    paused)
		if $work
		then
		    sudo virsh resume $domain
		    state=running
		fi
		;;
	    '')
		# probably not a VM, like "fuji"
		;;
	    *)
		echo "No clue what do with virsh state \"$state\""
		continue
		;;
	esac
    fi

    if $work
    then
	log=$HOME/Logs/by-date/$today/$domain
	touch $log
	mkdir -p $HOME/Logs/by-vm/$domain
	if [ ! -L $HOME/Logs/by-vm/$domain/$today ]
	then
	    ( cd $HOME/Logs/by-vm/$domain; ln -s ../../by-date/$today/$domain $today )
	fi

	rm -f $tmp.up
	wait=0
	delta=5
	max_wait=300
	while [ $wait -lt $max_wait ]
	do
	    if ssh $build_user@$host true >/dev/null 2>&1
	    then
		$debug && echo host $host up and ssh for $build_user ok
		if [ "$build_user" != "$qa_user" ]
		then
		    if ssh $qa_user@$host true
		    then
			$debug && echo ssh for $qa_user ok
		    else
			echo Warning cannot ssh for $qa_user to host $host
		    fi
		fi
		touch $tmp.up
		break
	    fi
	    $debug && echo host $host down, waiting $wait
	    sleep $delta
	    wait=`expr $wait + $delta`
	done
	if [ ! -f $tmp.up ]
	then
	    echo "Failed to start VM domain $domain after $max_wait secs ... giving up"
	    continue
	fi

	# per-host settings
	# MAKE=gmake for *BSD hosts
	#
	MAKE=make
	case "$osname"
	in
	    freebsd|netbsd|openbsd)
		MAKE=gmake
		;;
	esac
	$debug && echo "MAKE=$MAKE"

	_script_header daily >$tmp.script
	if [ -n "$efile" ]
	then
	    cat $efile >>$tmp.script
	fi
	_script_git >>$tmp.script
	$cflag && _script_git_check >>$tmp.script
	$pflag && _script_git_pull >>$tmp.script

	if $mflag
	then
	    _script_make >>$tmp.script
	fi

	$iflag && _script_install >>$tmp.script

	_script_header daily.qa >$tmp.script.qa

	cat >>$tmp.script.qa <<End-of-File

cd $qa_dir
End-of-File

	$qflag && _script_qa >>$tmp.script.qa
	$rflag && _script_qa_rerun >>$tmp.script.qa
	$hflag && _script_harvest >>$tmp.script.qa

	chmod 755 $tmp.script $tmp.script.qa
	# we've sometimes seen the scp fail because we get here a
	# bit too soon after a VM has been rebooted ... be a little
	# more patient
	#
	rm -f $tmp.ok
	for i in 1 2 3
	do
	    if scp $tmp.script $build_user@$host:daily.script >$tmp.out 2>&1
	    then
		touch $tmp.ok
		break
	    else
		cat $tmp.out
		echo "Warning: scp for daily.script failed!"
		sleep 2
	    fi
	done
	if [ ! -f $tmp.ok ]
	then
	    echo "Error: cannot copy daily.script to $host ... giving up"
	    continue
	fi

	if $qflag || $rflag || $hflag
	then
	    rm -f $tmp.ok
	    for i in 1 2 3
	    do
		if scp $tmp.script.qa $qa_user@$host:daily.qa.script >$tmp.out 2>&1
		then
		    touch $tmp.ok
		    break
		else
		    cat $tmp.out
		    echo "Warning: scp for daily.qa.script failed!"
		    sleep 2
		fi
	    done
	    if [ ! -f $tmp.ok ]
	    then
		echo "Error: cannot copy daily.qa.script to $host ... giving up"
		continue
	    fi
	fi

	# actually do the remote work here
	#
	start=`date +%s`
	ssh -t $build_user@$host sh -l -c ./daily.script
	scp -q $build_user@$host:daily.log $tmp.out
	echo "=== build and install: `date` ===" >>$log
	cat $tmp.out >>$log
	echo "=== END ===" >>$log

	if $qflag || $rflag || $hflag
	then
	    ssh -t $qa_user@$host sh -l -c ./daily.qa.script
	    scp -q $qa_user@$host:daily.qa.log $tmp.out
	    echo "=== qa: `date` ===" >>$log
	    cat $tmp.out >>$log
	    echo "=== END ===" >>$log
	fi

	if $iflag || $qflag || $rflag || $hflag
	then
	    # run whatami on QA host
	    ssh -t $qa_user@$host $qa_dir/admin/whatami >$HOME/Logs/by-vm/$domain/whatami
	fi

	if $hflag
	then
	    # harvest failures by pulling from QA host
	    echo "Harvest:"
	    if [ -d $HOME/Logs/by-vm/$domain/pcpqa ]
	    then
		# old dir name
		mv $HOME/Logs/by-vm/$domain/pcpqa $HOME/Logs/by-vm/$domain/qa
	    fi
	    if [ ! -d $HOME/Logs/by-vm/$domain/qa ]
	    then
		rm -f $HOME/Logs/by-vm/$domain/qa
		mkdir -p $HOME/Logs/by-vm/$domain/qa
		( cd $HOME/Logs/by-vm/$domain/qa;
		    for file in common common.rc common.check common.setup group show-me localconfig
		    do
		    	ln -s $HOME/src/pcp/qa/$file .
		    done
		)
	    fi
	    rm -f $HOME/Logs/by-vm/$domain/qa/[0-9]*
	    rm -f $tmp.harvest
	    scp $qa_user@$host:daily.harvest $tmp.harvest >$tmp.out 2>&1
	    if [ -s $tmp.harvest ]
	    then
		for file in `cat $tmp.harvest`
		do
		    echo -n " $file"
		    if scp $qa_user@$host:$qa_dir/$file $HOME/Logs/by-vm/$domain/qa >$tmp.err 2>&1
		    then
			:
		    else
			echo
			cat $tmp.err
		    fi
		done
		echo
	    elif [ -s $tmp.out ]
	    then
		echo "scp error ..."
		cat $tmp.out
	    else
		echo "No failures."
	    fi
	fi
	end=`date +%s`
	echo "$start $end" | awk >>$log '
    { delta = $2 - $1
      hr = int(delta/3600)
      min = int((delta-hr*3600)/60)
      sec = delta % 60
      printf "Elapsed time: %d:%02d:%02d (%d secs)\n", hr, min, sec, delta
    }'
    fi
    

    if $sflag
    then
	doit=false
	case $host
	in
	    vm01|vm03|vm15|vm35|vm36)
		# try to keep these VMs running, unless -f
		$fflag && doit=true
		;;
	    vm*)
		doit=true
		;;
	esac
	if $doit
	then
	    if [ $state = shut ]
	    then
		if $work
		then
		    echo "Warning: already shutdown"  | tee -a $HOME/daily.log
		else
		    echo "Warning: already shutdown"
		fi
	    else
		ssh -f -t -q $build_user@$host "sync; sleep 2; sync; export PATH=\$PATH:/sbin:/usr/sbin; ( sudo poweroff || sudo halt )" </dev/null &
		sleep 60
		sudo virsh shutdown $domain >/dev/null 2>&1
		sleep 10
		# seems a bit harsh, but we've given you two chances to
		# shutdown
		#
		sudo virsh destroy $domain
		if $work
		then
		    echo "Info: shutdown"  | tee -a $HOME/daily.log
		else
		    echo "Info: shutdown"
		fi
	    fi
	else
	    if $work
	    then
		echo "Warning: shutdown skipped"  | tee -a $HOME/daily.log
	    else
		echo "Warning: shutdown skipped"
	    fi
	fi
    fi

    $work && echo "Logs in $log"

done
